/*
 * Copyright 2009-present MongoDB, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <common-prelude.h>


#ifndef MONGO_C_DRIVER_COMMON_ATOMIC_PRIVATE_H
#define MONGO_C_DRIVER_COMMON_ATOMIC_PRIVATE_H


#include <bson/bson.h> // BSON_INLINE

#ifdef _MSC_VER
#include <intrin.h>
#endif


enum mcommon_memory_order {
   mcommon_memory_order_seq_cst,
   mcommon_memory_order_acquire,
   mcommon_memory_order_release,
   mcommon_memory_order_relaxed,
   mcommon_memory_order_acq_rel,
   mcommon_memory_order_consume,
};

#if defined(_M_ARM) /* MSVC memorder atomics are only avail on ARM */
#define MSVC_MEMORDER_SUFFIX(X) X
#else
#define MSVC_MEMORDER_SUFFIX(X)
#endif

#if defined(USE_LEGACY_GCC_ATOMICS) || (!defined(__clang__) && __GNUC__ == 4) || defined(__xlC__)
#define MCOMMON_USE_LEGACY_GCC_ATOMICS
#else
#undef MCOMMON_USE_LEGACY_GCC_ATOMICS
#endif

/* Not all GCC-like compilers support the current __atomic built-ins.  Older
 * GCC (pre-5) used different built-ins named with the __sync prefix.  When
 * compiling with such older GCC versions, it is necessary to use the applicable
 * functions, which requires redefining BSON_IF_GNU_LIKE and defining the
 * additional MCOMMON_IF_GNU_LEGACY_ATOMICS macro here. */
#ifdef MCOMMON_USE_LEGACY_GCC_ATOMICS
#undef BSON_IF_GNU_LIKE
#define BSON_IF_GNU_LIKE(...)
#define MCOMMON_IF_GNU_LEGACY_ATOMICS(...) __VA_ARGS__
#else
#define MCOMMON_IF_GNU_LEGACY_ATOMICS(...)
#endif

/* CDRIVER-4229 zSeries with gcc 4.8.4 produces illegal instructions for int and
 * int32 atomic intrinsics. */
#if defined(__s390__) || defined(__s390x__) || defined(__zarch__)
#define MCOMMON_EMULATE_INT32
#define MCOMMON_EMULATE_INT
#endif

/* CDRIVER-4264 Contrary to documentation, VS 2013 targeting x86 does not
 * correctly/consistently provide _InterlockedPointerExchange. */
#if defined(_MSC_VER) && _MSC_VER < 1900 && defined(_M_IX86)
#define MCOMMON_EMULATE_PTR
#endif

#define DEF_ATOMIC_OP(MSVC_Intrinsic, GNU_Intrinsic, GNU_Legacy_Intrinsic, Order, ...)              \
   do {                                                                                             \
      switch (Order) {                                                                              \
      case mcommon_memory_order_acq_rel:                                                            \
         BSON_IF_MSVC(return MSVC_Intrinsic(__VA_ARGS__);)                                          \
         BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_ACQ_REL);)                     \
         MCOMMON_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);)                   \
      case mcommon_memory_order_seq_cst:                                                            \
         BSON_IF_MSVC(return MSVC_Intrinsic(__VA_ARGS__);)                                          \
         BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_SEQ_CST);)                     \
         MCOMMON_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);)                   \
      case mcommon_memory_order_acquire:                                                            \
         BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_acq))(__VA_ARGS__);) \
         BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_ACQUIRE);)                     \
         MCOMMON_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);)                   \
      case mcommon_memory_order_consume:                                                            \
         BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_acq))(__VA_ARGS__);) \
         BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_CONSUME);)                     \
         MCOMMON_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);)                   \
      case mcommon_memory_order_release:                                                            \
         BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_rel))(__VA_ARGS__);) \
         BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_RELEASE);)                     \
         MCOMMON_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);)                   \
      case mcommon_memory_order_relaxed:                                                            \
         BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_nf))(__VA_ARGS__);)  \
         BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_RELAXED);)                     \
         MCOMMON_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);)                   \
      default:                                                                                      \
         BSON_UNREACHABLE("Invalid mcommon_memory_order value");                                    \
      }                                                                                             \
   } while (0)


#define DEF_ATOMIC_CMPEXCH_STRONG(VCSuffix1, VCSuffix2, GNU_MemOrder, Ptr, ExpectActualVar, NewValue)   \
   do {                                                                                                 \
      BSON_IF_MSVC(ExpectActualVar = BSON_CONCAT3(_InterlockedCompareExchange, VCSuffix1, VCSuffix2)(   \
                      Ptr, NewValue, ExpectActualVar);)                                                 \
      BSON_IF_GNU_LIKE((void)__atomic_compare_exchange_n(Ptr,                                           \
                                                         &ExpectActualVar,                              \
                                                         NewValue,                                      \
                                                         false, /* Not weak */                          \
                                                         GNU_MemOrder,                                  \
                                                         GNU_MemOrder);)                                \
      MCOMMON_IF_GNU_LEGACY_ATOMICS(__typeof__(ExpectActualVar) _val;                                   \
                                    _val = __sync_val_compare_and_swap(Ptr, ExpectActualVar, NewValue); \
                                    ExpectActualVar = _val;)                                            \
   } while (0)


#define DEF_ATOMIC_CMPEXCH_WEAK(VCSuffix1, VCSuffix2, GNU_MemOrder, Ptr, ExpectActualVar, NewValue)     \
   do {                                                                                                 \
      BSON_IF_MSVC(ExpectActualVar = BSON_CONCAT3(_InterlockedCompareExchange, VCSuffix1, VCSuffix2)(   \
                      Ptr, NewValue, ExpectActualVar);)                                                 \
      BSON_IF_GNU_LIKE((void)__atomic_compare_exchange_n(Ptr,                                           \
                                                         &ExpectActualVar,                              \
                                                         NewValue,                                      \
                                                         true, /* Yes weak */                           \
                                                         GNU_MemOrder,                                  \
                                                         GNU_MemOrder);)                                \
      MCOMMON_IF_GNU_LEGACY_ATOMICS(__typeof__(ExpectActualVar) _val;                                   \
                                    _val = __sync_val_compare_and_swap(Ptr, ExpectActualVar, NewValue); \
                                    ExpectActualVar = _val;)                                            \
   } while (0)


#define DECL_ATOMIC_INTEGRAL(NamePart, Type, VCIntrinSuffix)                                                           \
   static BSON_INLINE Type mcommon_atomic_##NamePart##_fetch_add(                                                      \
      Type volatile *a, Type addend, enum mcommon_memory_order ord)                                                    \
   {                                                                                                                   \
      DEF_ATOMIC_OP(BSON_CONCAT(_InterlockedExchangeAdd, VCIntrinSuffix),                                              \
                    __atomic_fetch_add,                                                                                \
                    __sync_fetch_and_add,                                                                              \
                    ord,                                                                                               \
                    a,                                                                                                 \
                    addend);                                                                                           \
   }                                                                                                                   \
                                                                                                                       \
   static BSON_INLINE Type mcommon_atomic_##NamePart##_fetch_sub(                                                      \
      Type volatile *a, Type subtrahend, enum mcommon_memory_order ord)                                                \
   {                                                                                                                   \
      /* MSVC doesn't have a subtract intrinsic, so just reuse addition    */                                          \
      BSON_IF_MSVC(return mcommon_atomic_##NamePart##_fetch_add(a, -subtrahend, ord);)                                 \
      BSON_IF_GNU_LIKE(DEF_ATOMIC_OP(~, __atomic_fetch_sub, ~, ord, a, subtrahend);)                                   \
      MCOMMON_IF_GNU_LEGACY_ATOMICS(DEF_ATOMIC_OP(~, ~, __sync_fetch_and_sub, ord, a, subtrahend);)                    \
   }                                                                                                                   \
                                                                                                                       \
   static BSON_INLINE Type mcommon_atomic_##NamePart##_fetch(Type volatile const *a, enum mcommon_memory_order order)  \
   {                                                                                                                   \
      /* MSVC doesn't have a load intrinsic, so just add zero */                                                       \
      BSON_IF_MSVC(return mcommon_atomic_##NamePart##_fetch_add((Type volatile *)a, 0, order);)                        \
      /* GNU doesn't want RELEASE order for the fetch operation, so we can't                                           \
       * just use DEF_ATOMIC_OP. */                                                                                    \
      BSON_IF_GNU_LIKE(switch (order) {                                                                                \
         case mcommon_memory_order_release: /* Fall back to seqcst */                                                  \
         case mcommon_memory_order_acq_rel: /* Fall back to seqcst */                                                  \
         case mcommon_memory_order_seq_cst:                                                                            \
            return __atomic_load_n(a, __ATOMIC_SEQ_CST);                                                               \
         case mcommon_memory_order_acquire:                                                                            \
            return __atomic_load_n(a, __ATOMIC_ACQUIRE);                                                               \
         case mcommon_memory_order_consume:                                                                            \
            return __atomic_load_n(a, __ATOMIC_CONSUME);                                                               \
         case mcommon_memory_order_relaxed:                                                                            \
            return __atomic_load_n(a, __ATOMIC_RELAXED);                                                               \
         default:                                                                                                      \
            BSON_UNREACHABLE("Invalid mcommon_memory_order value");                                                    \
      })                                                                                                               \
      MCOMMON_IF_GNU_LEGACY_ATOMICS({                                                                                  \
         BSON_UNUSED(order);                                                                                           \
         __sync_synchronize();                                                                                         \
         return *a;                                                                                                    \
      })                                                                                                               \
   }                                                                                                                   \
                                                                                                                       \
   static BSON_INLINE Type mcommon_atomic_##NamePart##_exchange(                                                       \
      Type volatile *a, Type value, enum mcommon_memory_order ord)                                                     \
   {                                                                                                                   \
      BSON_IF_MSVC(DEF_ATOMIC_OP(BSON_CONCAT(_InterlockedExchange, VCIntrinSuffix), ~, ~, ord, a, value);)             \
      /* GNU doesn't want CONSUME order for the exchange operation, so we                                              \
       * cannot use DEF_ATOMIC_OP. */                                                                                  \
      BSON_IF_GNU_LIKE(switch (ord) {                                                                                  \
         case mcommon_memory_order_acq_rel:                                                                            \
            return __atomic_exchange_n(a, value, __ATOMIC_ACQ_REL);                                                    \
         case mcommon_memory_order_release:                                                                            \
            return __atomic_exchange_n(a, value, __ATOMIC_RELEASE);                                                    \
         case mcommon_memory_order_seq_cst:                                                                            \
            return __atomic_exchange_n(a, value, __ATOMIC_SEQ_CST);                                                    \
         case mcommon_memory_order_consume: /* Fall back to acquire */                                                 \
         case mcommon_memory_order_acquire:                                                                            \
            return __atomic_exchange_n(a, value, __ATOMIC_ACQUIRE);                                                    \
         case mcommon_memory_order_relaxed:                                                                            \
            return __atomic_exchange_n(a, value, __ATOMIC_RELAXED);                                                    \
         default:                                                                                                      \
            BSON_UNREACHABLE("Invalid mcommon_memory_order value");                                                    \
      })                                                                                                               \
      MCOMMON_IF_GNU_LEGACY_ATOMICS(BSON_UNUSED(ord); return __sync_val_compare_and_swap(a, *a, value);)               \
   }                                                                                                                   \
                                                                                                                       \
   static BSON_INLINE Type mcommon_atomic_##NamePart##_compare_exchange_strong(                                        \
      Type volatile *a, Type expect, Type new_value, enum mcommon_memory_order ord)                                    \
   {                                                                                                                   \
      Type actual = expect;                                                                                            \
      switch (ord) {                                                                                                   \
      case mcommon_memory_order_release:                                                                               \
      case mcommon_memory_order_acq_rel:                                                                               \
      case mcommon_memory_order_seq_cst:                                                                               \
         DEF_ATOMIC_CMPEXCH_STRONG(VCIntrinSuffix, , __ATOMIC_SEQ_CST, a, actual, new_value);                          \
         break;                                                                                                        \
      case mcommon_memory_order_acquire:                                                                               \
         DEF_ATOMIC_CMPEXCH_STRONG(                                                                                    \
            VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_ACQUIRE, a, actual, new_value);                       \
         break;                                                                                                        \
      case mcommon_memory_order_consume:                                                                               \
         DEF_ATOMIC_CMPEXCH_STRONG(                                                                                    \
            VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_CONSUME, a, actual, new_value);                       \
         break;                                                                                                        \
      case mcommon_memory_order_relaxed:                                                                               \
         DEF_ATOMIC_CMPEXCH_STRONG(VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_nf), __ATOMIC_RELAXED, a, actual, new_value); \
         break;                                                                                                        \
      default:                                                                                                         \
         BSON_UNREACHABLE("Invalid mcommon_memory_order value");                                                       \
      }                                                                                                                \
      return actual;                                                                                                   \
   }                                                                                                                   \
                                                                                                                       \
   static BSON_INLINE Type mcommon_atomic_##NamePart##_compare_exchange_weak(                                          \
      Type volatile *a, Type expect, Type new_value, enum mcommon_memory_order ord)                                    \
   {                                                                                                                   \
      Type actual = expect;                                                                                            \
      switch (ord) {                                                                                                   \
      case mcommon_memory_order_release:                                                                               \
      case mcommon_memory_order_acq_rel:                                                                               \
      case mcommon_memory_order_seq_cst:                                                                               \
         DEF_ATOMIC_CMPEXCH_WEAK(VCIntrinSuffix, , __ATOMIC_SEQ_CST, a, actual, new_value);                            \
         break;                                                                                                        \
      case mcommon_memory_order_acquire:                                                                               \
         DEF_ATOMIC_CMPEXCH_WEAK(VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_ACQUIRE, a, actual, new_value);  \
         break;                                                                                                        \
      case mcommon_memory_order_consume:                                                                               \
         DEF_ATOMIC_CMPEXCH_WEAK(VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_CONSUME, a, actual, new_value);  \
         break;                                                                                                        \
      case mcommon_memory_order_relaxed:                                                                               \
         DEF_ATOMIC_CMPEXCH_WEAK(VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_nf), __ATOMIC_RELAXED, a, actual, new_value);   \
         break;                                                                                                        \
      default:                                                                                                         \
         BSON_UNREACHABLE("Invalid mcommon_memory_order value");                                                       \
      }                                                                                                                \
      return actual;                                                                                                   \
   }

#define DECL_ATOMIC_STDINT(Name, VCSuffix) DECL_ATOMIC_INTEGRAL(Name, Name##_t, VCSuffix)

#if defined(_MSC_VER) || defined(MCOMMON_USE_LEGACY_GCC_ATOMICS)
/* MSVC and GCC require built-in types (not typedefs) for their atomic
 * intrinsics. */
#if defined(_MSC_VER)
#define DECL_ATOMIC_INTEGRAL_INT8 char
#define DECL_ATOMIC_INTEGRAL_INT32 long
#define DECL_ATOMIC_INTEGRAL_INT long
#else
#define DECL_ATOMIC_INTEGRAL_INT8 signed char
#define DECL_ATOMIC_INTEGRAL_INT32 int
#define DECL_ATOMIC_INTEGRAL_INT int
#endif
DECL_ATOMIC_INTEGRAL(int8, DECL_ATOMIC_INTEGRAL_INT8, 8)
DECL_ATOMIC_INTEGRAL(int16, short, 16)
#if !defined(MCOMMON_EMULATE_INT32)
DECL_ATOMIC_INTEGRAL(int32, DECL_ATOMIC_INTEGRAL_INT32, )
#endif
#if !defined(MCOMMON_EMULATE_INT)
DECL_ATOMIC_INTEGRAL(int, DECL_ATOMIC_INTEGRAL_INT, )
#endif
#else
/* Other compilers that we support provide generic intrinsics */
DECL_ATOMIC_STDINT(int8, 8)
DECL_ATOMIC_STDINT(int16, 16)
#if !defined(MCOMMON_EMULATE_INT32)
DECL_ATOMIC_STDINT(int32, )
#endif
#if !defined(MCOMMON_EMULATE_INT)
DECL_ATOMIC_INTEGRAL(int, int, )
#endif
#endif

#ifndef DECL_ATOMIC_INTEGRAL_INT32
#define DECL_ATOMIC_INTEGRAL_INT32 int32_t
#endif

#define _mcommon_emul_atomic_int64_fetch_add COMMON_NAME(emul_atomic_int64_fetch_add)
#define _mcommon_emul_atomic_int64_exchange COMMON_NAME(emul_atomic_int64_exchange)
#define _mcommon_emul_atomic_int64_compare_exchange_strong COMMON_NAME(emul_atomic_int64_compare_exchange_strong)
#define _mcommon_emul_atomic_int64_compare_exchange_weak COMMON_NAME(emul_atomic_int64_compare_exchange_weak)
#define _mcommon_emul_atomic_int32_fetch_add COMMON_NAME(emul_atomic_int32_fetch_add)
#define _mcommon_emul_atomic_int32_exchange COMMON_NAME(emul_atomic_int32_exchange)
#define _mcommon_emul_atomic_int32_compare_exchange_strong COMMON_NAME(emul_atomic_int32_compare_exchange_strong)
#define _mcommon_emul_atomic_int32_compare_exchange_weak COMMON_NAME(emul_atomic_int32_compare_exchange_weak)
#define _mcommon_emul_atomic_int_fetch_add COMMON_NAME(emul_atomic_int_fetch_add)
#define _mcommon_emul_atomic_int_exchange COMMON_NAME(emul_atomic_int_exchange)
#define _mcommon_emul_atomic_int_compare_exchange_strong COMMON_NAME(emul_atomic_int_compare_exchange_strong)
#define _mcommon_emul_atomic_int_compare_exchange_weak COMMON_NAME(emul_atomic_int_compare_exchange_weak)
#define _mcommon_emul_atomic_ptr_exchange COMMON_NAME(emul_atomic_ptr_exchange)
#define mcommon_thrd_yield COMMON_NAME(thrd_yield)

int64_t
_mcommon_emul_atomic_int64_fetch_add(int64_t volatile *val, int64_t v, enum mcommon_memory_order);
int64_t
_mcommon_emul_atomic_int64_exchange(int64_t volatile *val, int64_t v, enum mcommon_memory_order);
int64_t
_mcommon_emul_atomic_int64_compare_exchange_strong(int64_t volatile *val,
                                                   int64_t expect_value,
                                                   int64_t new_value,
                                                   enum mcommon_memory_order);

int64_t
_mcommon_emul_atomic_int64_compare_exchange_weak(int64_t volatile *val,
                                                 int64_t expect_value,
                                                 int64_t new_value,
                                                 enum mcommon_memory_order);

int32_t
_mcommon_emul_atomic_int32_fetch_add(int32_t volatile *val, int32_t v, enum mcommon_memory_order);
int32_t
_mcommon_emul_atomic_int32_exchange(int32_t volatile *val, int32_t v, enum mcommon_memory_order);
int32_t
_mcommon_emul_atomic_int32_compare_exchange_strong(int32_t volatile *val,
                                                   int32_t expect_value,
                                                   int32_t new_value,
                                                   enum mcommon_memory_order);

int32_t
_mcommon_emul_atomic_int32_compare_exchange_weak(int32_t volatile *val,
                                                 int32_t expect_value,
                                                 int32_t new_value,
                                                 enum mcommon_memory_order);

int
_mcommon_emul_atomic_int_fetch_add(int volatile *val, int v, enum mcommon_memory_order);
int
_mcommon_emul_atomic_int_exchange(int volatile *val, int v, enum mcommon_memory_order);
int
_mcommon_emul_atomic_int_compare_exchange_strong(int volatile *val,
                                                 int expect_value,
                                                 int new_value,
                                                 enum mcommon_memory_order);

int
_mcommon_emul_atomic_int_compare_exchange_weak(int volatile *val,
                                               int expect_value,
                                               int new_value,
                                               enum mcommon_memory_order);

void *
_mcommon_emul_atomic_ptr_exchange(void *volatile *val, void *v, enum mcommon_memory_order);

void
mcommon_thrd_yield(void);

#if (defined(_MSC_VER) && !defined(_M_IX86)) || (defined(__LP64__) && __LP64__)
/* (64-bit intrinsics are only available in x64) */
#ifdef _MSC_VER
DECL_ATOMIC_INTEGRAL(int64, __int64, 64)
#else
DECL_ATOMIC_STDINT(int64, 64)
#endif
#else
static BSON_INLINE int64_t
mcommon_atomic_int64_fetch(const int64_t volatile *val, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int64_fetch_add((int64_t volatile *)val, 0, order);
}

static BSON_INLINE int64_t
mcommon_atomic_int64_fetch_add(int64_t volatile *val, int64_t v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int64_fetch_add(val, v, order);
}

static BSON_INLINE int64_t
mcommon_atomic_int64_fetch_sub(int64_t volatile *val, int64_t v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int64_fetch_add(val, -v, order);
}

static BSON_INLINE int64_t
mcommon_atomic_int64_exchange(int64_t volatile *val, int64_t v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int64_exchange(val, v, order);
}

static BSON_INLINE int64_t
mcommon_atomic_int64_compare_exchange_strong(int64_t volatile *val,
                                             int64_t expect_value,
                                             int64_t new_value,
                                             enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int64_compare_exchange_strong(val, expect_value, new_value, order);
}

static BSON_INLINE int64_t
mcommon_atomic_int64_compare_exchange_weak(int64_t volatile *val,
                                           int64_t expect_value,
                                           int64_t new_value,
                                           enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int64_compare_exchange_weak(val, expect_value, new_value, order);
}
#endif

#if defined(MCOMMON_EMULATE_INT32)
static BSON_INLINE int32_t
mcommon_atomic_int32_fetch(const int32_t volatile *val, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int32_fetch_add((int32_t volatile *)val, 0, order);
}

static BSON_INLINE int32_t
mcommon_atomic_int32_fetch_add(int32_t volatile *val, int32_t v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int32_fetch_add(val, v, order);
}

static BSON_INLINE int32_t
mcommon_atomic_int32_fetch_sub(int32_t volatile *val, int32_t v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int32_fetch_add(val, -v, order);
}

static BSON_INLINE int32_t
mcommon_atomic_int32_exchange(int32_t volatile *val, int32_t v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int32_exchange(val, v, order);
}

static BSON_INLINE int32_t
mcommon_atomic_int32_compare_exchange_strong(int32_t volatile *val,
                                             int32_t expect_value,
                                             int32_t new_value,
                                             enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int32_compare_exchange_strong(val, expect_value, new_value, order);
}

static BSON_INLINE int32_t
mcommon_atomic_int32_compare_exchange_weak(int32_t volatile *val,
                                           int32_t expect_value,
                                           int32_t new_value,
                                           enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int32_compare_exchange_weak(val, expect_value, new_value, order);
}
#endif /* MCOMMON_EMULATE_INT32 */

#if defined(MCOMMON_EMULATE_INT)
static BSON_INLINE int
mcommon_atomic_int_fetch(const int volatile *val, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int_fetch_add((int volatile *)val, 0, order);
}

static BSON_INLINE int
mcommon_atomic_int_fetch_add(int volatile *val, int v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int_fetch_add(val, v, order);
}

static BSON_INLINE int
mcommon_atomic_int_fetch_sub(int volatile *val, int v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int_fetch_add(val, -v, order);
}

static BSON_INLINE int
mcommon_atomic_int_exchange(int volatile *val, int v, enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int_exchange(val, v, order);
}

static BSON_INLINE int
mcommon_atomic_int_compare_exchange_strong(int volatile *val,
                                           int expect_value,
                                           int new_value,
                                           enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int_compare_exchange_strong(val, expect_value, new_value, order);
}

static BSON_INLINE int
mcommon_atomic_int_compare_exchange_weak(int volatile *val,
                                         int expect_value,
                                         int new_value,
                                         enum mcommon_memory_order order)
{
   return _mcommon_emul_atomic_int_compare_exchange_weak(val, expect_value, new_value, order);
}
#endif /* MCOMMON_EMULATE_INT */

static BSON_INLINE void *
mcommon_atomic_ptr_exchange(void *volatile *ptr, void *new_value, enum mcommon_memory_order ord)
{
#if defined(MCOMMON_EMULATE_PTR)
   return _mcommon_emul_atomic_ptr_exchange(ptr, new_value, ord);
#elif defined(MCOMMON_USE_LEGACY_GCC_ATOMICS)
   /* The older __sync_val_compare_and_swap also takes oldval */
   DEF_ATOMIC_OP(_InterlockedExchangePointer, , __sync_val_compare_and_swap, ord, ptr, *ptr, new_value);
#else
   DEF_ATOMIC_OP(_InterlockedExchangePointer, __atomic_exchange_n, , ord, ptr, new_value);
#endif
}

static BSON_INLINE void *
mcommon_atomic_ptr_compare_exchange_strong(void *volatile *ptr,
                                           void *expect,
                                           void *new_value,
                                           enum mcommon_memory_order ord)
{
   switch (ord) {
   case mcommon_memory_order_release:
   case mcommon_memory_order_acq_rel:
   case mcommon_memory_order_seq_cst:
      DEF_ATOMIC_CMPEXCH_STRONG(Pointer, , __ATOMIC_SEQ_CST, ptr, expect, new_value);
      return expect;
   case mcommon_memory_order_relaxed:
      DEF_ATOMIC_CMPEXCH_STRONG(Pointer, MSVC_MEMORDER_SUFFIX(_nf), __ATOMIC_RELAXED, ptr, expect, new_value);
      return expect;
   case mcommon_memory_order_consume:
      DEF_ATOMIC_CMPEXCH_STRONG(Pointer, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_CONSUME, ptr, expect, new_value);
      return expect;
   case mcommon_memory_order_acquire:
      DEF_ATOMIC_CMPEXCH_STRONG(Pointer, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_ACQUIRE, ptr, expect, new_value);
      return expect;
   default:
      BSON_UNREACHABLE("Invalid mcommon_memory_order value");
   }
}


static BSON_INLINE void *
mcommon_atomic_ptr_compare_exchange_weak(void *volatile *ptr,
                                         void *expect,
                                         void *new_value,
                                         enum mcommon_memory_order ord)
{
   switch (ord) {
   case mcommon_memory_order_release:
   case mcommon_memory_order_acq_rel:
   case mcommon_memory_order_seq_cst:
      DEF_ATOMIC_CMPEXCH_WEAK(Pointer, , __ATOMIC_SEQ_CST, ptr, expect, new_value);
      return expect;
   case mcommon_memory_order_relaxed:
      DEF_ATOMIC_CMPEXCH_WEAK(Pointer, MSVC_MEMORDER_SUFFIX(_nf), __ATOMIC_RELAXED, ptr, expect, new_value);
      return expect;
   case mcommon_memory_order_consume:
      DEF_ATOMIC_CMPEXCH_WEAK(Pointer, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_CONSUME, ptr, expect, new_value);
      return expect;
   case mcommon_memory_order_acquire:
      DEF_ATOMIC_CMPEXCH_WEAK(Pointer, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_ACQUIRE, ptr, expect, new_value);
      return expect;
   default:
      BSON_UNREACHABLE("Invalid mcommon_memory_order value");
   }
}


static BSON_INLINE void *
mcommon_atomic_ptr_fetch(void *volatile const *ptr, enum mcommon_memory_order ord)
{
   return mcommon_atomic_ptr_compare_exchange_strong((void *volatile *)ptr, NULL, NULL, ord);
}

#undef DECL_ATOMIC_STDINT
#undef DECL_ATOMIC_INTEGRAL
#undef DEF_ATOMIC_OP
#undef DEF_ATOMIC_CMPEXCH_STRONG
#undef DEF_ATOMIC_CMPEXCH_WEAK
#undef MSVC_MEMORDER_SUFFIX

/**
 * @brief Generate a full-fence memory barrier at the call site.
 */
static BSON_INLINE void
mcommon_atomic_thread_fence(void)
{
   BSON_IF_MSVC(MemoryBarrier();)
   BSON_IF_GNU_LIKE(__sync_synchronize();)
   MCOMMON_IF_GNU_LEGACY_ATOMICS(__sync_synchronize();)
}

#ifdef MCOMMON_USE_LEGACY_GCC_ATOMICS
#undef BSON_IF_GNU_LIKE
#define BSON_IF_GNU_LIKE(...) __VA_ARGS__
#endif
#undef MCOMMON_IF_GNU_LEGACY_ATOMICS
#undef MCOMMON_USE_LEGACY_GCC_ATOMICS


#undef MCOMMON_EMULATE_PTR
#undef MCOMMON_EMULATE_INT32
#undef MCOMMON_EMULATE_INT


#endif /* MONGO_C_DRIVER_COMMON_ATOMIC_PRIVATE_H */
